package com.yym.cloudweb.crawl.station.dadianjing;

import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.htmlcleaner.XPatherException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.seimicrawler.xpath.JXDocument;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;

import java.util.List;

public class DianjinghuPageProcessor implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);

    @Override
    public void process(Page page) {
        String html = page.getHtml().toString();
        String xpath = "//div[@class='list-left article-left']/a";
        JXDocument jxDocument = JXDocument.create(html);
        List<Object> rs = jxDocument.sel(xpath);
        for(Object object : rs){
            if(object instanceof Element){
                Element element = (Element)object;
                String artLink = getValByXpath(element,"//a/@href");
                String coverImg = getValByXpath(element,"//img/@src");
                String title = getValByXpath(element,"//div/h4/text()");
                String desc = getValByXpath(element,"//div/p/text()");

                //Spider.create(new DianjinghuDetailPageProcessor()).addUrl(artLink).setDownloader(Downloader.getInstance()).thread(1).run();

            }
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        //Spider.create(new DianjinghuPageProcessor()).addUrl("https://www.dianjinghu.com/news/p/all/2.html").setDownloader(Downloader.getInstance()).thread(5).run();
    }

    private static String getValByXpath(Element element, String xpath){
        JXDocument eleJxDocument = JXDocument.create(element.toString());
        return eleJxDocument.selOne(xpath).toString();
    }

}
